package exp.baseline;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import tools.FileWriterEnhance;
import tools.ObjectTools;
import tools.twitter.TweetTidy;

import com.maruti.otterapi.search.Post;

public class ReadResult4MEAD {
	public static final String topic = "Obama inauguration";
	public static final String path = "data/objs/topsyCrawl/" + topic + "/";
	
	public static void main(String[] args) {
		StringBuilder clusterFile = new StringBuilder();
		clusterFile.append("<?xml version='1.0'?>\n<CLUSTER LANG='ENG'>\n");
		
		List<Post> allPosts = new ArrayList<>();
		for (File file : (new File(path)).listFiles()) {
			@SuppressWarnings("unchecked")
			List<Post> posts = (List<Post>) ObjectTools.readFromFile(file);
			allPosts.addAll(posts);
		}
		HashMap<String, Boolean> outputed = new HashMap<>();
		int docID = 1;
		for (Post post : allPosts) {
			String content = post.getContent().trim();
			if (content.length() > 140 || content.length() == 0) {
				continue;
			}
			// whether we need to delete the duplicate tweets 
			if (outputed.containsKey(content)) {
				continue;
			} else {
				outputed.put(content, true);
			}
//			String docIDString = String.format("%06d",docID);
			String docIDString = String.format("%d",docID);
			clusterFile.append("\t<D DID='");
			clusterFile.append(docIDString);
			clusterFile.append("' />\n");
			
			StringBuilder docFile = new StringBuilder();
			content = TweetTidy.doTidyAll(post.getContent().replaceAll("\\s+", " "));
			content = content.replaceAll(">", "&gt;");
			content = content.replaceAll("<", "&lt;");
			content = content.replaceAll("\"", "&quot;");
			content = content.replaceAll("'", "&apos;");
			content = content.replaceAll("&", "&amp;");
			docFile.append("<?xml version='1.0' encoding='UTF-8'?>\n<!DOCTYPE DOCSENT SYSTEM \"/clair/tools/MEAD3/dtd/docsent.dtd\">\n<DOCSENT DID='" 
					+ docID + "' DOCNO='" 
					+ docID + "' LANG='ENG' CORR-DOC='" 
					+ docID + ".c'>\n<BODY>\n<HEADLINE>\n<S PAR='1' RSNT='1' SNO='1'>");
			docFile.append(content);
			docFile.append("</S>\n</HEADLINE>\n<TEXT>\n");
			docFile.append("<S PAR='2' RSNT='1' SNO='2'>");
			docFile.append(content);
			docFile.append("</S>\n</TEXT>\n</BODY>\n</DOCSENT>\n");
			FileWriterEnhance fwe = new FileWriterEnhance("data/_newData/MEAD/" + topic +  "/docsent/" + docIDString + ".docsent", "UTF-8");
			fwe.WriteToFile(docFile.toString());
			docID ++;
//			if (docID > 20000) {
//				break;
//			}
		}
		
		clusterFile.append("</CLUSTER>\n");
		FileWriterEnhance fwe = new FileWriterEnhance("data/_newData/MEAD/" + topic + "/" + topic + ".cluster", "UTF-8");
		fwe.WriteToFile(clusterFile.toString());
	}
}
